package excute;

import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;
import schedual.PnSchedual;
import schedual.Schedual;
import schedual.UrlSchedual;
import utiles.HbaseUtils;
import utiles.MD5Utils;

import java.io.IOException;

public class Spilder {
    private HttpClient httpClient = HttpClients.createDefault();
    private HtmlCleaner cleaner =new HtmlCleaner();
    private Schedual pnSchedual = new PnSchedual();
    private Schedual urlSchedual = new UrlSchedual();

    //下载网页
    private String downLoad(String url) throws IOException {
        HttpGet httpGet = new HttpGet(url);
        HttpResponse resp = httpClient.execute(httpGet);
        String html = EntityUtils.toString(resp.getEntity());
        return html;
    }
    //解析网页
    private Object[] parseHtml(String html,String xPath) throws XPatherException {
        TagNode tagNode = cleaner.clean(html);
        Object[] o = tagNode.evaluateXPath(xPath);
        return o;
    }

    //爬取网页的url
    public void spildUrl(String url,String pn) throws IOException, XPatherException {
        String html = downLoad(url+pn);
        Object[] objects = parseHtml(html, "//a[@class='j_th_tit ']");
        for (Object o : objects) {
            TagNode tagNode = (TagNode)o;
            String href = tagNode.getAttributeByName("href");
            urlSchedual.push("urls",href);
        }
        //下一页
        int pnInt = Integer.parseInt(pn);
        String pnString = pnInt+50+"";
        pnSchedual.push("pn",pnString);
        String pnNext = pnSchedual.poll("pn");
        if(pnInt<=200000)
            System.out.println(pnInt);
            spildUrl(url,pnNext);
    }

    //爬取网页标题以及图片
    public void spildContent(String url) throws Exception {
        String html = downLoad("https://tieba.baidu.com"+url);
        String title = parseHtml(html, "//h3/text()")[0].toString();
        //行键的设计（MD5(标题+时间戳)+","+"标题"）
        String rowkey = MD5Utils.MD5(title+System.currentTimeMillis())+","+title;
      //存入标题
        HbaseUtils.save(rowkey,"title",title);
        Object[] images = parseHtml(html, "//img[@class='BDE_Image']");
        if (images.length!=0) {
            StringBuffer imageString = new StringBuffer();
            for (Object image : images) {
                TagNode imageNode = (TagNode)image;
                String src = imageNode.getAttributeByName("src");
                imageString.append(","+src);
                System.out.println(src);
//                FileUtils.downLoadPict(src,src.substring(src.lastIndexOf("/"),src.length()),"e:/data/picture/");
            }
            //存入图片地址
            HbaseUtils.save(rowkey,"image",imageString.toString());
        }
    }
}
